/* -------------------------------------------------------------------------
 *
 * dict_thesaurus.c
 *		Thesaurus dictionary: phrase to phrase substitution
 *
 * Portions Copyright (c) 1996-2012, PostgreSQL Global Development Group
 *
 *
 * IDENTIFICATION
 *	  src/backend/tsearch/dict_thesaurus.c
 *
 * -------------------------------------------------------------------------
 */
#include "postgres.h"
#include "knl/knl_variable.h"

#include "catalog/namespace.h"
#include "commands/defrem.h"
#include "tsearch/ts_cache.h"
#include "tsearch/ts_locale.h"
#include "tsearch/ts_utils.h"
#include "utils/builtins.h"

/*
 * Temporay we use TSLexeme.flags for inner use...
 */
#define DT_USEASIS 0x1000

typedef struct LexemeInfo {
    uint32 idsubst;    /* entry's number in DictThesaurus->subst */
    uint16 posinsubst; /* pos info in entry */
    uint16 tnvariant;  /* total num lexemes in one variant */
    struct LexemeInfo* nextentry;
    struct LexemeInfo* nextvariant;
} LexemeInfo;

typedef struct {
    char* lexeme;
    LexemeInfo* entries;
} TheLexeme;

typedef struct {
    uint16 lastlexeme; /* number lexemes to substitute */
    uint16 reslen;
    TSLexeme* res; /* prepared substituted result */
} TheSubstitute;

typedef struct {
    /* subdictionary to normalize lexemes */
    Oid subdictOid;
    TSDictionaryCacheEntry* subdict;

    /* Array to search lexeme by exact match */
    TheLexeme* wrds;
    int nwrds;  /* current number of words */
    int ntwrds; /* allocated array length */

    /*
     * Storage of substituted result, n-th element is for n-th expression
     */
    TheSubstitute* subst;
    int nsubst;
} DictThesaurus;

static void newLexeme(DictThesaurus* d, char* b, char* e, uint32 idsubst, uint16 posinsubst)
{
    TheLexeme* ptr = NULL;
    errno_t errorno = EOK;

    if (d->nwrds >= d->ntwrds) {
        if (d->ntwrds == 0) {
            d->ntwrds = 16;
            d->wrds = (TheLexeme*)palloc(sizeof(TheLexeme) * d->ntwrds);
        } else {
            d->ntwrds *= 2;
            d->wrds = (TheLexeme*)repalloc(d->wrds, sizeof(TheLexeme) * d->ntwrds);
        }
    }

    ptr = d->wrds + d->nwrds;
    d->nwrds++;

    ptr->lexeme = (char*)palloc(e - b + 1);

    errorno = memcpy_s(ptr->lexeme, e - b, b, e - b);
    securec_check(errorno, "\0", "\0");
    ptr->lexeme[e - b] = '\0';

    ptr->entries = (LexemeInfo*)palloc(sizeof(LexemeInfo));

    ptr->entries->nextentry = NULL;
    ptr->entries->idsubst = idsubst;
    ptr->entries->posinsubst = posinsubst;
}

static void addWrd(DictThesaurus* d, char* b, char* e, uint32 idsubst, uint16 nwrd, uint16 posinsubst, bool useasis)
{
    TheSubstitute* ptr = NULL;

    if (nwrd == 0) {
        t_thrd.tsearch_cxt.nres = t_thrd.tsearch_cxt.ntres = 0;

        if (idsubst >= (uint32)d->nsubst) {
            if (d->nsubst == 0) {
                d->nsubst = 16;
                d->subst = (TheSubstitute*)palloc(sizeof(TheSubstitute) * d->nsubst);
            } else {
                d->nsubst *= 2;
                d->subst = (TheSubstitute*)repalloc(d->subst, sizeof(TheSubstitute) * d->nsubst);
            }
        }
    }

    ptr = d->subst + idsubst;

    ptr->lastlexeme = posinsubst - 1;

    if (t_thrd.tsearch_cxt.nres + 1 >= t_thrd.tsearch_cxt.ntres) {
        if (t_thrd.tsearch_cxt.ntres == 0) {
            t_thrd.tsearch_cxt.ntres = 2;
            ptr->res = (TSLexeme*)palloc(sizeof(TSLexeme) * t_thrd.tsearch_cxt.ntres);
        } else {
            t_thrd.tsearch_cxt.ntres *= 2;
            ptr->res = (TSLexeme*)repalloc(ptr->res, sizeof(TSLexeme) * t_thrd.tsearch_cxt.ntres);
        }
    }
    errno_t errorno = EOK;

    ptr->res[t_thrd.tsearch_cxt.nres].lexeme = (char*)palloc(e - b + 1);
    errorno = memcpy_s(ptr->res[t_thrd.tsearch_cxt.nres].lexeme, e - b, b, e - b);
    securec_check(errorno, "\0", "\0");
    ptr->res[t_thrd.tsearch_cxt.nres].lexeme[e - b] = '\0';

    ptr->res[t_thrd.tsearch_cxt.nres].nvariant = nwrd;
    if (useasis) {
        ptr->res[t_thrd.tsearch_cxt.nres].flags = DT_USEASIS;
    } else {
        ptr->res[t_thrd.tsearch_cxt.nres].flags = 0;
    }

    ptr->res[++t_thrd.tsearch_cxt.nres].lexeme = NULL;
}

#define TR_WAITLEX 1
#define TR_INLEX 2
#define TR_WAITSUBS 3
#define TR_INSUBS 4

static void thesaurusRead(char* filename, DictThesaurus* d)
{
    tsearch_readline_state trst;
    uint32 idsubst = 0;
    bool useasis = false;
    char* line = NULL;

    if (!tsearch_readline_begin(&trst, filename))
        ereport(
            ERROR, (errcode(ERRCODE_CONFIG_FILE_ERROR), errmsg("could not open thesaurus file \"%s\": %m", filename)));

    while ((line = tsearch_readline(&trst)) != NULL) {
        char* ptr = NULL;
        int state = TR_WAITLEX;
        char* beginwrd = NULL;
        uint32 posinsubst = 0;
        uint32 nwrd = 0;

        ptr = line;

        /* is it a comment? */
        while (*ptr && t_isspace(ptr))
            ptr += pg_mblen(ptr);

        if (t_iseq(ptr, '#') || *ptr == '\0' || t_iseq(ptr, '\n') || t_iseq(ptr, '\r')) {
            pfree_ext(line);
            continue;
        }

        while (*ptr) {
            if (state == TR_WAITLEX) {
                if (t_iseq(ptr, ':')) {
                    if (posinsubst == 0)
                        ereport(ERROR, (errcode(ERRCODE_CONFIG_FILE_ERROR), errmsg("unexpected delimiter")));
                    state = TR_WAITSUBS;
                } else if (!t_isspace(ptr)) {
                    beginwrd = ptr;
                    state = TR_INLEX;
                }
            } else if (state == TR_INLEX) {
                if (t_iseq(ptr, ':')) {
                    newLexeme(d, beginwrd, ptr, idsubst, posinsubst++);
                    state = TR_WAITSUBS;
                } else if (t_isspace(ptr)) {
                    newLexeme(d, beginwrd, ptr, idsubst, posinsubst++);
                    state = TR_WAITLEX;
                }
            } else if (state == TR_WAITSUBS) {
                if (t_iseq(ptr, '*')) {
                    useasis = true;
                    state = TR_INSUBS;
                    beginwrd = ptr + pg_mblen(ptr);
                } else if (t_iseq(ptr, '\\')) {
                    useasis = false;
                    state = TR_INSUBS;
                    beginwrd = ptr + pg_mblen(ptr);
                } else if (!t_isspace(ptr)) {
                    useasis = false;
                    beginwrd = ptr;
                    state = TR_INSUBS;
                }
            } else if (state == TR_INSUBS) {
                if (t_isspace(ptr)) {
                    if (ptr == beginwrd)
                        ereport(
                            ERROR, (errcode(ERRCODE_CONFIG_FILE_ERROR), errmsg("unexpected end of line or lexeme")));
                    addWrd(d, beginwrd, ptr, idsubst, nwrd++, posinsubst, useasis);
                    state = TR_WAITSUBS;
                }
            } else
                ereport(
                    ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("unrecognized thesaurus state: %d", state)));

            ptr += pg_mblen(ptr);
        }

        if (state == TR_INSUBS) {
            if (ptr == beginwrd)
                ereport(ERROR, (errcode(ERRCODE_CONFIG_FILE_ERROR), errmsg("unexpected end of line or lexeme")));
            addWrd(d, beginwrd, ptr, idsubst, nwrd++, posinsubst, useasis);
        }

        idsubst++;

        if (!(nwrd && posinsubst))
            ereport(ERROR, (errcode(ERRCODE_CONFIG_FILE_ERROR), errmsg("unexpected end of line")));

        /*
         * Note: currently, tsearch_readline can't return lines exceeding 4KB,
         * so overflow of the word counts is impossible.  But that may not
         * always be true, so let's check.
         */
        if (nwrd != (uint16)nwrd || posinsubst != (uint16)posinsubst)
            ereport(ERROR, (errcode(ERRCODE_CONFIG_FILE_ERROR), errmsg("too many lexemes in thesaurus entry")));

        pfree_ext(line);
    }

    d->nsubst = idsubst;

    tsearch_readline_end(&trst);
}

static TheLexeme* addCompiledLexeme(
    TheLexeme* newwrds, int* nnw, int* tnm, TSLexeme* lexeme, LexemeInfo* src, uint16 tnvariant)
{
    if (*nnw >= *tnm) {
        *tnm *= 2;
        newwrds = (TheLexeme*)repalloc(newwrds, sizeof(TheLexeme) * *tnm);
    }

    newwrds[*nnw].entries = (LexemeInfo*)palloc(sizeof(LexemeInfo));

    if (lexeme != NULL && lexeme->lexeme != NULL) {
        newwrds[*nnw].lexeme = pstrdup(lexeme->lexeme);
        newwrds[*nnw].entries->tnvariant = tnvariant;
    } else {
        newwrds[*nnw].lexeme = NULL;
        newwrds[*nnw].entries->tnvariant = 1;
    }

    newwrds[*nnw].entries->idsubst = src->idsubst;
    newwrds[*nnw].entries->posinsubst = src->posinsubst;

    newwrds[*nnw].entries->nextentry = NULL;

    (*nnw)++;
    return newwrds;
}

static int cmpLexemeInfo(LexemeInfo* a, LexemeInfo* b)
{
    if (a == NULL || b == NULL)
        return 0;

    if (a->idsubst == b->idsubst) {
        if (a->posinsubst == b->posinsubst) {
            if (a->tnvariant == b->tnvariant)
                return 0;

            return (a->tnvariant > b->tnvariant) ? 1 : -1;
        }

        return (a->posinsubst > b->posinsubst) ? 1 : -1;
    }

    return (a->idsubst > b->idsubst) ? 1 : -1;
}

static int cmpLexeme(const TheLexeme* a, const TheLexeme* b)
{
    if (a->lexeme == NULL) {
        if (b->lexeme == NULL)
            return 0;
        else
            return 1;
    } else if (b->lexeme == NULL)
        return -1;

    return strcmp(a->lexeme, b->lexeme);
}

static int cmpLexemeQ(const void* a, const void* b)
{
    return cmpLexeme((const TheLexeme*)a, (const TheLexeme*)b);
}

static int cmpTheLexeme(const void* a, const void* b)
{
    const TheLexeme* la = (const TheLexeme*)a;
    const TheLexeme* lb = (const TheLexeme*)b;
    int res;

    if ((res = cmpLexeme(la, lb)) != 0) {
        return res;
    }

    return -cmpLexemeInfo(la->entries, lb->entries);
}

static void compileTheLexeme(DictThesaurus* d)
{
    int i;
    int nnw = 0;
    int tnm = 16;
    TheLexeme *newwrds = (TheLexeme*)palloc(sizeof(TheLexeme) * tnm);
    TheLexeme *ptrwrds = NULL;

    for (i = 0; i < d->nwrds; i++) {
        TSLexeme* ptr = NULL;

        /* Is stop word marker? */
        if (strcmp(d->wrds[i].lexeme, "?") == 0) {
            newwrds = addCompiledLexeme(newwrds, &nnw, &tnm, NULL, d->wrds[i].entries, 0);
        } else {
            ptr = (TSLexeme*)DatumGetPointer(FunctionCall4(&(d->subdict->lexize),
                PointerGetDatum(d->subdict->dictData),
                PointerGetDatum(d->wrds[i].lexeme),
                Int32GetDatum(strlen(d->wrds[i].lexeme)),
                PointerGetDatum(NULL)));
            if (ptr == NULL) {
                ereport(ERROR,
                    (errcode(ERRCODE_CONFIG_FILE_ERROR),
                        errmsg("thesaurus sample word \"%s\" isn't recognized by subdictionary (rule %u)",
                            d->wrds[i].lexeme,
                            d->wrds[i].entries->idsubst + 1)));
            } else if (!(ptr->lexeme)) {
                ereport(ERROR,
                    (errcode(ERRCODE_CONFIG_FILE_ERROR),
                        errmsg("thesaurus sample word \"%s\" is a stop word (rule %u)",
                            d->wrds[i].lexeme,
                            d->wrds[i].entries->idsubst + 1),
                        errhint("Use \"?\" to represent a stop word within a sample phrase.")));
            } else {
                while (ptr->lexeme) {
                    TSLexeme* remptr = ptr + 1;
                    int tnvar = 1;
                    int curvar = ptr->nvariant;

                    /* compute n words in one variant */
                    while (remptr->lexeme) {
                        if (remptr->nvariant != (remptr - 1)->nvariant)
                            break;
                        tnvar++;
                        remptr++;
                    }

                    remptr = ptr;
                    while (remptr->lexeme && remptr->nvariant == curvar) {
                        newwrds = addCompiledLexeme(newwrds, &nnw, &tnm, remptr, d->wrds[i].entries, tnvar);
                        remptr++;
                    }

                    ptr = remptr;
                }
            }
        }

        pfree_ext(d->wrds[i].lexeme);
        pfree_ext(d->wrds[i].entries);
    }

    pfree_ext(d->wrds);
    d->wrds = newwrds;
    d->nwrds = nnw;
    d->ntwrds = tnm;

    if (d->nwrds > 1) {
        qsort(d->wrds, d->nwrds, sizeof(TheLexeme), cmpTheLexeme);

        /* uniq */
        newwrds = d->wrds;
        ptrwrds = d->wrds + 1;
        while (ptrwrds - d->wrds < d->nwrds) {
            if (cmpLexeme(ptrwrds, newwrds) == 0) {
                if (cmpLexemeInfo(ptrwrds->entries, newwrds->entries)) {
                    ptrwrds->entries->nextentry = newwrds->entries;
                    newwrds->entries = ptrwrds->entries;
                } else {
                    pfree_ext(ptrwrds->entries);
                }

                if (ptrwrds->lexeme != NULL) {
                    pfree_ext(ptrwrds->lexeme);
                }
            } else {
                newwrds++;
                *newwrds = *ptrwrds;
            }

            ptrwrds++;
        }

        d->nwrds = newwrds - d->wrds + 1;
        d->wrds = (TheLexeme*)repalloc(d->wrds, sizeof(TheLexeme) * d->nwrds);
    }
}

static void compileTheSubstitute(DictThesaurus* d)
{
    int i;

    for (i = 0; i < d->nsubst; i++) {
        TSLexeme *rem = d->subst[i].res;
        TSLexeme *outptr = NULL;
        TSLexeme *inptr = NULL;
        int n = 2;

        outptr = d->subst[i].res = (TSLexeme*)palloc(sizeof(TSLexeme) * n);
        outptr->lexeme = NULL;
        inptr = rem;

        while (inptr != NULL && inptr->lexeme != NULL) {
            TSLexeme *lexized = NULL;
            TSLexeme tmplex[2];

            if (inptr->flags & DT_USEASIS) { /* do not lexize */
                tmplex[0] = *inptr;
                tmplex[0].flags = 0;
                tmplex[1].lexeme = NULL;
                lexized = tmplex;
            } else {
                lexized = (TSLexeme*)DatumGetPointer(FunctionCall4(&(d->subdict->lexize),
                    PointerGetDatum(d->subdict->dictData),
                    PointerGetDatum(inptr->lexeme),
                    Int32GetDatum(strlen(inptr->lexeme)),
                    PointerGetDatum(NULL)));
            }

            if (lexized != NULL && lexized->lexeme != NULL) {
                int toset = (lexized->lexeme && outptr != d->subst[i].res) ? (outptr - d->subst[i].res) : -1;

                while (lexized->lexeme) {
                    if (outptr - d->subst[i].res + 1 >= n) {
                        int diff = outptr - d->subst[i].res;

                        n *= 2;
                        d->subst[i].res = (TSLexeme*)repalloc(d->subst[i].res, sizeof(TSLexeme) * n);
                        outptr = d->subst[i].res + diff;
                    }

                    *outptr = *lexized;
                    outptr->lexeme = pstrdup(lexized->lexeme);

                    outptr++;
                    lexized++;
                }

                if (toset > 0) {
                    d->subst[i].res[toset].flags |= TSL_ADDPOS;
                }
            } else if (lexized != NULL) {
                ereport(ERROR,
                    (errcode(ERRCODE_CONFIG_FILE_ERROR),
                        errmsg("thesaurus substitute word \"%s\" is a stop word (rule %d)", inptr->lexeme, i + 1)));
            } else {
                ereport(ERROR,
                    (errcode(ERRCODE_CONFIG_FILE_ERROR),
                        errmsg("thesaurus substitute word \"%s\" isn't recognized by subdictionary (rule %d)",
                            inptr->lexeme,
                            i + 1)));
            }

            if (inptr->lexeme) {
                pfree_ext(inptr->lexeme);
            }
            inptr++;
        }

        if (outptr == d->subst[i].res) {
            ereport(ERROR,
                (errcode(ERRCODE_CONFIG_FILE_ERROR), errmsg("thesaurus substitute phrase is empty (rule %d)", i + 1)));
        }

        d->subst[i].reslen = outptr - d->subst[i].res;

        pfree_ext(rem);
    }
}

Datum thesaurus_init(PG_FUNCTION_ARGS)
{
    List* dictoptions = (List*)PG_GETARG_POINTER(0);
    ListCell* l = NULL;

    /* 3 params in dictoptions */
    char* thsfile = NULL;
    bool thsloaded = false;
    bool thschanged = false;
    char* subdictname = NULL;
    char* pathname = NULL;
    bool pathloaded = false;

    /* if we are create/alter dictionary */
    bool ddlsql = (3 == PG_NARGS());
    char* dictprefix = NULL;
    int oldnum = list_length(dictoptions);
    int optidx = 0;

    if (ddlsql) {
        dictprefix = PG_GETARG_CSTRING(1);
        oldnum = PG_GETARG_INT32(2);
        if (dictprefix == NULL || oldnum < 0) {
            ereport(ERROR, (errcode(ERRCODE_SYSTEM_ERROR), errmsg("Init dictionary failed for invalid inputs")));
        }
    }

    foreach (l, dictoptions) {
        DefElem* defel = (DefElem*)lfirst(l);

        if (pg_strcasecmp("DictFile", defel->defname) == 0) {
            if (thsloaded != false) {
                ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("multiple DictFile parameters")));
            }

            thsfile = defGetString(defel);
            thsloaded = true;

            /*
             * if create/alter dictionary, replace the user-defined
             * file name with the internal one.
             */
            if (ddlsql) {
                defel->arg = (Node*)makeString(dictprefix);
                if (optidx >= oldnum) {
                    thschanged = true;
                }
            }
        } else if (pg_strcasecmp("Dictionary", defel->defname) == 0) {
            if (subdictname != NULL) {
                ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("multiple Dictionary parameters")));
            }

            subdictname = pstrdup(defGetString(defel));
        } else if (pg_strcasecmp("FilePath", defel->defname) == 0) {
            if (pathloaded) {
                ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("multiple FilePath parameters")));
            }

            /* user defined filepath */
            pathname = defGetString(defel);
            pathloaded = true;
        } else {
            ereport(ERROR,
                (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
                    errmsg("unrecognized Thesaurus parameter: \"%s\"", defel->defname)));
        }

        optidx++;
    }

    if (!thsloaded) {
        ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("missing DictFile parameter")));
    }

    if (subdictname == NULL) {
        ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("missing Dictionary parameter")));
    }

    /* Not allowed filepath without dictionary file when create/alter */
    if (pathname != NULL && !thschanged) {
        ereport(
            ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("FilePath parameter should be with DictFile")));
    }

    /* Get full file name with absolute path */
    List* userfiles = NIL;
    char* absfname = get_tsearch_config_filename(thsfile, pathname, FILE_POSTFIX_THS, ddlsql && thschanged);
    if (ddlsql) {
        userfiles = lappend(userfiles, absfname);
    }

    /*
     * Need to load dictionary for two cases:
     * 1.Use dictionary
     * 2.Create/alter dictionary query from user directly
     */
    DictThesaurus* d = NULL;
    if (!skipLoad(ddlsql)) {
        d = (DictThesaurus*)palloc0(sizeof(DictThesaurus));
        thesaurusRead(absfname, d);

        List* result = stringToQualifiedNameList(subdictname);
        d->subdictOid = get_ts_dict_oid(result, false);
        d->subdict = lookup_ts_dictionary_cache(d->subdictOid);
        list_free(result);

        compileTheLexeme(d);
        compileTheSubstitute(d);
    }

    if (ddlsql) {
        PG_RETURN_POINTER(userfiles);
    } else {
        PG_RETURN_POINTER(d);
    }
}

static LexemeInfo* findTheLexeme(DictThesaurus* d, char* lexeme)
{
    TheLexeme key;
    TheLexeme *res = NULL;

    if (d->nwrds == 0) {
        return NULL;
    }

    key.lexeme = lexeme;
    key.entries = NULL;

    res = (TheLexeme*)bsearch(&key, d->wrds, d->nwrds, sizeof(TheLexeme), cmpLexemeQ);
    if (res == NULL) {
        return NULL;
    }
    return res->entries;
}

static bool matchIdSubst(LexemeInfo* stored, uint32 idsubst)
{
    bool res = true;

    if (stored != NULL) {
        res = false;

        for (; stored; stored = stored->nextvariant) {
            if (stored->idsubst == idsubst) {
                res = true;
                break;
            }
        }
    }

    return res;
}

static LexemeInfo* findVariant(LexemeInfo* in, LexemeInfo* stored, uint16 curpos, LexemeInfo** newin, int newn)
{
    for (;;) {
        int i;
        LexemeInfo* ptr = newin[0];

        for (i = 0; i < newn; i++) {
            while (newin[i] && newin[i]->idsubst < ptr->idsubst)
                newin[i] = newin[i]->nextentry;

            if (newin[i] == NULL) {
                return in;
            }

            if (newin[i]->idsubst > ptr->idsubst) {
                ptr = newin[i];
                i = -1;
                continue;
            }

            while (newin[i]->idsubst == ptr->idsubst) {
                if (newin[i]->posinsubst == curpos && newin[i]->tnvariant == newn) {
                    ptr = newin[i];
                    break;
                }

                newin[i] = newin[i]->nextentry;
                if (newin[i] == NULL) {
                    return in;
                }
            }

            if (newin[i]->idsubst != ptr->idsubst) {
                ptr = newin[i];
                i = -1;
                continue;
            }
        }

        if (i == newn && matchIdSubst(stored, ptr->idsubst) &&
            (in == NULL || !matchIdSubst(in, ptr->idsubst))) { /* found */
            ptr->nextvariant = in;
            in = ptr;
        }

        /* step forward */
        for (i = 0; i < newn; i++) {
            newin[i] = newin[i]->nextentry;
        }
    }

    return NULL;
}

static TSLexeme* copyTSLexeme(TheSubstitute* ts)
{
    TSLexeme* res = NULL;
    uint16 i;

    res = (TSLexeme*)palloc(sizeof(TSLexeme) * (ts->reslen + 1));
    for (i = 0; i < ts->reslen; i++) {
        res[i] = ts->res[i];
        res[i].lexeme = pstrdup(ts->res[i].lexeme);
    }

    res[ts->reslen].lexeme = NULL;

    return res;
}

static TSLexeme* checkMatch(DictThesaurus* d, LexemeInfo* info, uint16 curpos, bool* moreres)
{
    *moreres = false;
    while (info != NULL) {
        Assert(info->idsubst < (uint32)d->nsubst);
        if (info->nextvariant != NULL)
            *moreres = true;
        if (d->subst[info->idsubst].lastlexeme == curpos) {
            return copyTSLexeme(d->subst + info->idsubst);
        }
        info = info->nextvariant;
    }

    return NULL;
}

Datum thesaurus_lexize(PG_FUNCTION_ARGS)
{
    DictThesaurus* d = (DictThesaurus*)PG_GETARG_POINTER(0);
    DictSubState* dstate = (DictSubState*)PG_GETARG_POINTER(3);
    TSLexeme* res = NULL;
    LexemeInfo *stored = NULL;
    LexemeInfo *info = NULL;
    uint16 curpos = 0;
    bool moreres = false;

    if (PG_NARGS() != 4 || dstate == NULL) {
        ereport(ERROR,
            (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("forbidden call of thesaurus or nested call")));
    }

    if (dstate->isend) {
        PG_RETURN_POINTER(NULL);
    }
    stored = (LexemeInfo*)dstate->private_state;

    if (stored != NULL) {
        curpos = stored->posinsubst + 1;
    }

    if (!d->subdict->isvalid) {
        d->subdict = lookup_ts_dictionary_cache(d->subdictOid);
    }

    res = (TSLexeme*)DatumGetPointer(FunctionCall4(&(d->subdict->lexize),
        PointerGetDatum(d->subdict->dictData),
        PG_GETARG_DATUM(1),
        PG_GETARG_DATUM(2),
        PointerGetDatum(NULL)));
    if (res != NULL && res->lexeme != NULL) {
        TSLexeme *ptr = res;
        TSLexeme *basevar = NULL;

        while (ptr->lexeme) {
            uint16 nv = ptr->nvariant;
            uint16 i;
            uint16 nlex = 0;
            LexemeInfo** infos;

            basevar = ptr;
            while (ptr->lexeme && nv == ptr->nvariant) {
                nlex++;
                ptr++;
            }

            infos = (LexemeInfo**)palloc(sizeof(LexemeInfo*) * nlex);
            for (i = 0; i < nlex; i++)
                if ((infos[i] = findTheLexeme(d, basevar[i].lexeme)) == NULL) {
                    break;
                }

            if (i < nlex) {
                /* no chance to find */
                pfree_ext(infos);
                continue;
            }

            info = findVariant(info, stored, curpos, infos, nlex);
        }
    } else if (res != NULL) { /* stop-word */
        LexemeInfo* infos = findTheLexeme(d, NULL);

        info = findVariant(NULL, stored, curpos, &infos, 1);
    } else {
        info = NULL; /* word isn't recognized */
    }

    dstate->private_state = (void*)info;

    if (info == NULL) {
        dstate->getnext = false;
        PG_RETURN_POINTER(NULL);
    }

    if ((res = checkMatch(d, info, curpos, &moreres)) != NULL) {
        dstate->getnext = moreres;
        PG_RETURN_POINTER(res);
    }

    dstate->getnext = true;

    PG_RETURN_POINTER(NULL);
}

